package com.qwiki.xml;

import java.util.HashSet;

import org.apache.log4j.Logger;

import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;

public class WikiPageAsyncWordDocFreqHandler extends WikiPageAbstractAsyncHandler {
  protected final Logger _log = Logger.getLogger(WikiPageAsyncWordDocFreqHandler.class);
  private WikiXMLPageFilter filter;
  public final Object2IntOpenHashMap<String> counters = new Object2IntOpenHashMap<String>();

  /**
   * @param name
   * @param id
   */
  public WikiPageAsyncWordDocFreqHandler(String name, int id) {
    super(name, id);
    counters.defaultReturnValue(0);
  }

  /*
   * (non-Javadoc)
   *
   * @see
   * com.qwiki.xml.WikiPageAbstractAsyncHandler#doProcess(com.qwiki.xml.WikiXMLPage
   * )
   */
  @Override
  public void doProcess(WikiXMLPage xmlpage) {
    if (filter == null || filter.isGood(xmlpage)) {
      try {
        WikiPage page = xmlpage.parse();
        String text = page.getText().toLowerCase();
        text = text.replaceAll("[^a-zA-Z]", " ");
        text = text.replaceAll("  ", " ");
        HashSet<String> dedup = new HashSet<String>();
        String[] tokens = text.split(" ");
        for(String token : tokens) {
          if (token.length() > 3 && !dedup.contains(token)) {
            counters.put(token, counters.getInt(token) + 1);
            dedup.add(token);
          }
        }
      } catch (Exception e) {
        _log.error("error processing " + xmlpage.getTitle(), e);
      }
    }
  }

  /*
   * (non-Javadoc)
   *
   * @see com.qwiki.xml.WikiPageHandler#getFilter()
   */
  @Override
  public synchronized WikiXMLPageFilter getFilter() {
    // TODO Auto-generated method stub
    return filter;
  }

  /*
   * (non-Javadoc)
   *
   * @see
   * com.qwiki.xml.WikiPageHandler#setFilter(com.qwiki.xml.WikiXMLPageFilter)
   */
  @Override
  public synchronized void setFilter(WikiXMLPageFilter filter) {
    this.filter = filter;
  }

}
